package PreProcessData;

import java.util.HashMap;
import java.util.Map;
import java.io.BufferedReader;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;

import Classes.Path;
/**
 * This is for INFSCI-2140 in 2020
 *
 * The class for trecweb format
 */
public class TrecwebCollection implements DocumentCollection {
	
	// YOU CAN ADD ESSENTIAL PRIVATE METHODS OR VARIABLES -> 
	public static final String HTML_TAG_PATTERN = "(<(.*?)>)|(\\[(.*?)\\])|(\\&\\#?\\S+( ))|(( )\\S+>( ))";
	
	private BufferedReader br = null;
	private StringBuilder sb = null;
	
	// YOU SHOULD IMPLEMENT THIS METHOD ->
	public TrecwebCollection() throws IOException {
		// This constructor should open the file existing in Path.DataWebDir
		// and also should make preparation for function nextDocument().
		// You cannot load the whole corpus into memory here!!!
		FileInputStream fis = new FileInputStream(Path.DataWebDir); // Path.DataWebDir | "test/test_trecweb"
		InputStreamReader isr = new InputStreamReader(fis);
		br = new BufferedReader(isr);
	}

	// YOU SHOULD IMPLEMENT THIS METHOD ->
	public Map<String, Object> nextDocument() throws IOException {
		// This method should load one document from the corpus, and return this document's number and content.
		// The returned document should never be returned again.
		// When no document left, return null.
		// Note: The returned content of the document should be cleaned, all html tags should be removed.
		// Remember to close the file that you opened, when you do not use it anymore.
		Map<String, Object> webDoc = new HashMap<>(); // Store all docs with their ID and content
		String line = ""; // Every line read by BufferedReader
		String pureLine = ""; // Every line with only text 
		String webDocID = ""; // The doc's number 
		String webDocContent = ""; // The doc's content
		
		while((line = br.readLine()) != null) { // Continue reading a line while the end of stream has not been reached
			
			if("<DOC>".equals(line)) { // If "<DOC>" is found
				line = br.readLine(); // Read the next line containing "<DOCNO>"
				webDocID = line.substring(7, 24);
				//System.out.println("webDocID: " + webDocID);
			} 
			
			if("</DOCHDR>".equals(line)) {
				sb = new StringBuilder();
				while(!("</DOC>").equals(line = br.readLine())) {
					pureLine = line.replaceAll(HTML_TAG_PATTERN, " ");
					//System.out.println("pureLine: " + pureLine);
					sb.append(pureLine);		
				}	
				webDocContent = sb.toString();
			} 
			
			if("</DOC>".equals(line)) {
				//System.out.println("webDocContent:" + webDocContent);
				webDoc.put(webDocID, webDocContent.replaceAll(HTML_TAG_PATTERN, " ")
												  .replaceAll("( )+", " ")
												  .toCharArray());
				return webDoc;	
			}		
		}
	    
		br.close();
		return null;
	}
}